{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The MIT License (MIT)\n",
    "\n",
    "# Copyright (c) 2020, NVIDIA CORPORATION.\n",
    "\n",
    "# Permission is hereby granted, free of charge, to any person obtaining a copy of\n",
    "# this software and associated documentation files (the \"Software\"), to deal in\n",
    "# the Software without restriction, including without limitation the rights to\n",
    "# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of\n",
    "# the Software, and to permit persons to whom the Software is furnished to do so,\n",
    "# subject to the following conditions:\n",
    "\n",
    "# The above copyright notice and this permission notice shall be included in all\n",
    "# copies or substantial portions of the Software.\n",
    "\n",
    "# THE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n",
    "# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS\n",
    "# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR\n",
    "# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER\n",
    "# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN\n",
    "# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tutorial: Feature Engineering for Recommender Systems\n",
    "\n",
    "# 3. Feature Engineering - Categorical\n",
    "\n",
    "## 3.3. Target Encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import IPython\n",
    "\n",
    "import pandas as pd\n",
    "import cudf\n",
    "import numpy as np\n",
    "import cupy\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "df_train = cudf.read_parquet('../data/train.parquet')\n",
    "df_valid = cudf.read_parquet('../data/valid.parquet')\n",
    "df_test = cudf.read_parquet('../data/test.parquet')\n",
    "\n",
    "df_train['brand'] = df_train['brand'].fillna('UNKNOWN')\n",
    "df_valid['brand'] = df_valid['brand'].fillna('UNKNOWN')\n",
    "df_test['brand'] = df_test['brand'].fillna('UNKNOWN')\n",
    "df_train['cat_2'] = df_train['cat_2'].fillna('UNKNOWN')\n",
    "df_valid['cat_2'] = df_valid['cat_2'].fillna('UNKNOWN')\n",
    "df_test['cat_2'] = df_test['cat_2'].fillna('UNKNOWN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event_time</th>\n",
       "      <th>event_type</th>\n",
       "      <th>product_id</th>\n",
       "      <th>brand</th>\n",
       "      <th>price</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_session</th>\n",
       "      <th>target</th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>ts_hour</th>\n",
       "      <th>ts_minute</th>\n",
       "      <th>ts_weekday</th>\n",
       "      <th>ts_day</th>\n",
       "      <th>ts_month</th>\n",
       "      <th>ts_year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-12-01 00:00:28 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>17800342</td>\n",
       "      <td>zeta</td>\n",
       "      <td>66.90</td>\n",
       "      <td>550465671</td>\n",
       "      <td>22650a62-2d9c-4151-9f41-2674ec6d32d5</td>\n",
       "      <td>0</td>\n",
       "      <td>computers</td>\n",
       "      <td>desktop</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 00:00:28</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-12-01 00:00:39 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>3701309</td>\n",
       "      <td>polaris</td>\n",
       "      <td>89.32</td>\n",
       "      <td>543733099</td>\n",
       "      <td>a65116f4-ac53-4a41-ad68-6606788e674c</td>\n",
       "      <td>0</td>\n",
       "      <td>appliances</td>\n",
       "      <td>environment</td>\n",
       "      <td>vacuum</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 00:00:39</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-12-01 00:00:40 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>3701309</td>\n",
       "      <td>polaris</td>\n",
       "      <td>89.32</td>\n",
       "      <td>543733099</td>\n",
       "      <td>a65116f4-ac53-4a41-ad68-6606788e674c</td>\n",
       "      <td>0</td>\n",
       "      <td>appliances</td>\n",
       "      <td>environment</td>\n",
       "      <td>vacuum</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 00:00:40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-12-01 00:00:41 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>3701309</td>\n",
       "      <td>polaris</td>\n",
       "      <td>89.32</td>\n",
       "      <td>543733099</td>\n",
       "      <td>a65116f4-ac53-4a41-ad68-6606788e674c</td>\n",
       "      <td>0</td>\n",
       "      <td>appliances</td>\n",
       "      <td>environment</td>\n",
       "      <td>vacuum</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 00:00:41</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-12-01 00:01:56 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>1004767</td>\n",
       "      <td>samsung</td>\n",
       "      <td>235.60</td>\n",
       "      <td>579970209</td>\n",
       "      <td>c6946211-ce70-4228-95ce-fd7fccdde63c</td>\n",
       "      <td>0</td>\n",
       "      <td>construction</td>\n",
       "      <td>tools</td>\n",
       "      <td>light</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 00:01:56</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                event_time event_type  product_id    brand   price    user_id  \\\n",
       "0  2019-12-01 00:00:28 UTC       cart    17800342     zeta   66.90  550465671   \n",
       "1  2019-12-01 00:00:39 UTC       cart     3701309  polaris   89.32  543733099   \n",
       "2  2019-12-01 00:00:40 UTC       cart     3701309  polaris   89.32  543733099   \n",
       "3  2019-12-01 00:00:41 UTC       cart     3701309  polaris   89.32  543733099   \n",
       "4  2019-12-01 00:01:56 UTC       cart     1004767  samsung  235.60  579970209   \n",
       "\n",
       "                           user_session  target         cat_0        cat_1  \\\n",
       "0  22650a62-2d9c-4151-9f41-2674ec6d32d5       0     computers      desktop   \n",
       "1  a65116f4-ac53-4a41-ad68-6606788e674c       0    appliances  environment   \n",
       "2  a65116f4-ac53-4a41-ad68-6606788e674c       0    appliances  environment   \n",
       "3  a65116f4-ac53-4a41-ad68-6606788e674c       0    appliances  environment   \n",
       "4  c6946211-ce70-4228-95ce-fd7fccdde63c       0  construction        tools   \n",
       "\n",
       "     cat_2 cat_3            timestamp  ts_hour  ts_minute  ts_weekday  ts_day  \\\n",
       "0  UNKNOWN  <NA>  2019-12-01 00:00:28        0          0           6       1   \n",
       "1   vacuum  <NA>  2019-12-01 00:00:39        0          0           6       1   \n",
       "2   vacuum  <NA>  2019-12-01 00:00:40        0          0           6       1   \n",
       "3   vacuum  <NA>  2019-12-01 00:00:41        0          0           6       1   \n",
       "4    light  <NA>  2019-12-01 00:01:56        0          1           6       1   \n",
       "\n",
       "   ts_month  ts_year  \n",
       "0        12     2019  \n",
       "1        12     2019  \n",
       "2        12     2019  \n",
       "3        12     2019  \n",
       "4        12     2019  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat = 'brand'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Theory"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>*Target Encoding (TE)*</b> calculates the statistics from a target variable grouped by the unique values of one or more categorical features.<br><br>\n",
    "For example in a binary classification problem, it calculates the probability that the target is true for each category value - a simple mean."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "te = df_train[[cat, 'target']].groupby(cat).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>brand</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>UNKNOWN</th>\n",
       "      <td>0.301577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a-case</th>\n",
       "      <td>0.264910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a-derma</th>\n",
       "      <td>0.150442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a-elita</th>\n",
       "      <td>0.275862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a-mega</th>\n",
       "      <td>0.340426</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zuru</th>\n",
       "      <td>0.285714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zvezda</th>\n",
       "      <td>0.444444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zwilling</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zwillingjahenckels</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zyxel</th>\n",
       "      <td>0.285714</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4638 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      target\n",
       "brand                       \n",
       "UNKNOWN             0.301577\n",
       "a-case              0.264910\n",
       "a-derma             0.150442\n",
       "a-elita             0.275862\n",
       "a-mega              0.340426\n",
       "...                      ...\n",
       "zuru                0.285714\n",
       "zvezda              0.444444\n",
       "zwilling            0.000000\n",
       "zwillingjahenckels  0.000000\n",
       "zyxel               0.285714\n",
       "\n",
       "[4638 rows x 1 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "te"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event_time</th>\n",
       "      <th>event_type</th>\n",
       "      <th>product_id</th>\n",
       "      <th>brand</th>\n",
       "      <th>price</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_session</th>\n",
       "      <th>target</th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>ts_hour</th>\n",
       "      <th>ts_minute</th>\n",
       "      <th>ts_weekday</th>\n",
       "      <th>ts_day</th>\n",
       "      <th>ts_month</th>\n",
       "      <th>ts_year</th>\n",
       "      <th>TE_brand</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-12-01 12:27:02 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>12700214</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>35.38</td>\n",
       "      <td>580243411</td>\n",
       "      <td>0cbf5e06-a782-4c74-8002-acf282026d82</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 12:27:02</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.301577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-12-01 12:27:02 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>12700214</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>35.38</td>\n",
       "      <td>580243411</td>\n",
       "      <td>0cbf5e06-a782-4c74-8002-acf282026d82</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 12:27:02</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.301577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-12-01 12:27:02 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>12700214</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>35.38</td>\n",
       "      <td>580243411</td>\n",
       "      <td>0cbf5e06-a782-4c74-8002-acf282026d82</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 12:27:02</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.301577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-12-01 12:27:02 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>12700214</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>35.38</td>\n",
       "      <td>580243411</td>\n",
       "      <td>0cbf5e06-a782-4c74-8002-acf282026d82</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 12:27:02</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.301577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-12-01 12:27:02 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>12700214</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>35.38</td>\n",
       "      <td>580243411</td>\n",
       "      <td>0cbf5e06-a782-4c74-8002-acf282026d82</td>\n",
       "      <td>0</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 12:27:02</td>\n",
       "      <td>12</td>\n",
       "      <td>27</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.301577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461352</th>\n",
       "      <td>2019-11-30 19:09:17 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>1004856</td>\n",
       "      <td>samsung</td>\n",
       "      <td>124.11</td>\n",
       "      <td>514132559</td>\n",
       "      <td>56cf6962-e2bf-4d31-a45a-7b87257b0b2a</td>\n",
       "      <td>1</td>\n",
       "      <td>electronics</td>\n",
       "      <td>smartphone</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 19:09:17</td>\n",
       "      <td>19</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.439618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461353</th>\n",
       "      <td>2019-11-30 19:09:19 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>4804056</td>\n",
       "      <td>apple</td>\n",
       "      <td>160.87</td>\n",
       "      <td>522760118</td>\n",
       "      <td>772f04a5-80c2-4d15-99ce-0eb45a26b384</td>\n",
       "      <td>1</td>\n",
       "      <td>electronics</td>\n",
       "      <td>audio</td>\n",
       "      <td>headphone</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 19:09:19</td>\n",
       "      <td>19</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.421482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461354</th>\n",
       "      <td>2019-11-30 19:09:21 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>12711507</td>\n",
       "      <td>tunga</td>\n",
       "      <td>45.56</td>\n",
       "      <td>512586698</td>\n",
       "      <td>d8d092e4-d7c0-42ed-ac2b-783020d509db</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 19:09:21</td>\n",
       "      <td>19</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.210910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461355</th>\n",
       "      <td>2019-11-30 19:09:24 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>4803976</td>\n",
       "      <td>samsung</td>\n",
       "      <td>123.22</td>\n",
       "      <td>572105640</td>\n",
       "      <td>222dd50e-42ef-40da-93d2-679944ae9921</td>\n",
       "      <td>1</td>\n",
       "      <td>electronics</td>\n",
       "      <td>audio</td>\n",
       "      <td>headphone</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 19:09:24</td>\n",
       "      <td>19</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.439618</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461356</th>\n",
       "      <td>2019-11-30 19:09:25 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>9300087</td>\n",
       "      <td>sony</td>\n",
       "      <td>205.64</td>\n",
       "      <td>513956227</td>\n",
       "      <td>0f1c71a5-b4ac-4773-aac7-d223cc7352a8</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 19:09:25</td>\n",
       "      <td>19</td>\n",
       "      <td>9</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.351208</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11461357 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       event_time event_type  product_id    brand   price  \\\n",
       "0         2019-12-01 12:27:02 UTC       cart    12700214  UNKNOWN   35.38   \n",
       "1         2019-12-01 12:27:02 UTC       cart    12700214  UNKNOWN   35.38   \n",
       "2         2019-12-01 12:27:02 UTC       cart    12700214  UNKNOWN   35.38   \n",
       "3         2019-12-01 12:27:02 UTC       cart    12700214  UNKNOWN   35.38   \n",
       "4         2019-12-01 12:27:02 UTC       cart    12700214  UNKNOWN   35.38   \n",
       "...                           ...        ...         ...      ...     ...   \n",
       "11461352  2019-11-30 19:09:17 UTC   purchase     1004856  samsung  124.11   \n",
       "11461353  2019-11-30 19:09:19 UTC   purchase     4804056    apple  160.87   \n",
       "11461354  2019-11-30 19:09:21 UTC   purchase    12711507    tunga   45.56   \n",
       "11461355  2019-11-30 19:09:24 UTC   purchase     4803976  samsung  123.22   \n",
       "11461356  2019-11-30 19:09:25 UTC   purchase     9300087     sony  205.64   \n",
       "\n",
       "            user_id                          user_session  target  \\\n",
       "0         580243411  0cbf5e06-a782-4c74-8002-acf282026d82       0   \n",
       "1         580243411  0cbf5e06-a782-4c74-8002-acf282026d82       0   \n",
       "2         580243411  0cbf5e06-a782-4c74-8002-acf282026d82       0   \n",
       "3         580243411  0cbf5e06-a782-4c74-8002-acf282026d82       0   \n",
       "4         580243411  0cbf5e06-a782-4c74-8002-acf282026d82       0   \n",
       "...             ...                                   ...     ...   \n",
       "11461352  514132559  56cf6962-e2bf-4d31-a45a-7b87257b0b2a       1   \n",
       "11461353  522760118  772f04a5-80c2-4d15-99ce-0eb45a26b384       1   \n",
       "11461354  512586698  d8d092e4-d7c0-42ed-ac2b-783020d509db       1   \n",
       "11461355  572105640  222dd50e-42ef-40da-93d2-679944ae9921       1   \n",
       "11461356  513956227  0f1c71a5-b4ac-4773-aac7-d223cc7352a8       1   \n",
       "\n",
       "                cat_0       cat_1      cat_2 cat_3            timestamp  \\\n",
       "0                <NA>        <NA>    UNKNOWN  <NA>  2019-12-01 12:27:02   \n",
       "1                <NA>        <NA>    UNKNOWN  <NA>  2019-12-01 12:27:02   \n",
       "2                <NA>        <NA>    UNKNOWN  <NA>  2019-12-01 12:27:02   \n",
       "3                <NA>        <NA>    UNKNOWN  <NA>  2019-12-01 12:27:02   \n",
       "4                <NA>        <NA>    UNKNOWN  <NA>  2019-12-01 12:27:02   \n",
       "...               ...         ...        ...   ...                  ...   \n",
       "11461352  electronics  smartphone    UNKNOWN  <NA>  2019-11-30 19:09:17   \n",
       "11461353  electronics       audio  headphone  <NA>  2019-11-30 19:09:19   \n",
       "11461354         <NA>        <NA>    UNKNOWN  <NA>  2019-11-30 19:09:21   \n",
       "11461355  electronics       audio  headphone  <NA>  2019-11-30 19:09:24   \n",
       "11461356         <NA>        <NA>    UNKNOWN  <NA>  2019-11-30 19:09:25   \n",
       "\n",
       "          ts_hour  ts_minute  ts_weekday  ts_day  ts_month  ts_year  TE_brand  \n",
       "0              12         27           6       1        12     2019  0.301577  \n",
       "1              12         27           6       1        12     2019  0.301577  \n",
       "2              12         27           6       1        12     2019  0.301577  \n",
       "3              12         27           6       1        12     2019  0.301577  \n",
       "4              12         27           6       1        12     2019  0.301577  \n",
       "...           ...        ...         ...     ...       ...      ...       ...  \n",
       "11461352       19          9           5      30        11     2019  0.439618  \n",
       "11461353       19          9           5      30        11     2019  0.421482  \n",
       "11461354       19          9           5      30        11     2019  0.210910  \n",
       "11461355       19          9           5      30        11     2019  0.439618  \n",
       "11461356       19          9           5      30        11     2019  0.351208  \n",
       "\n",
       "[11461357 rows x 20 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "te = te.reset_index()\n",
    "te.columns = [cat, 'TE_' + cat]\n",
    "df_train.merge(te, how='left', on=cat)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Similarly, we can apply Target Encoding to a group of categorical features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "te = df_train[['brand', 'cat_2', 'target']].groupby(['brand', 'cat_2']).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>brand</th>\n",
       "      <th>cat_2</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">UNKNOWN</th>\n",
       "      <th>UNKNOWN</th>\n",
       "      <td>0.277106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>acoustic</th>\n",
       "      <td>0.294273</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>air_conditioner</th>\n",
       "      <td>0.152439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>air_heater</th>\n",
       "      <td>0.281630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>alarm</th>\n",
       "      <td>0.324934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zwilling</th>\n",
       "      <th>kettle</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">zwillingjahenckels</th>\n",
       "      <th>UNKNOWN</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kettle</th>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">zyxel</th>\n",
       "      <th>mouse</th>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>table</th>\n",
       "      <td>0.250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11154 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      target\n",
       "brand              cat_2                    \n",
       "UNKNOWN            UNKNOWN          0.277106\n",
       "                   acoustic         0.294273\n",
       "                   air_conditioner  0.152439\n",
       "                   air_heater       0.281630\n",
       "                   alarm            0.324934\n",
       "...                                      ...\n",
       "zwilling           kettle           0.000000\n",
       "zwillingjahenckels UNKNOWN          0.000000\n",
       "                   kettle           0.000000\n",
       "zyxel              mouse            0.333333\n",
       "                   table            0.250000\n",
       "\n",
       "[11154 rows x 1 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "te"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event_time</th>\n",
       "      <th>event_type</th>\n",
       "      <th>product_id</th>\n",
       "      <th>brand</th>\n",
       "      <th>price</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_session</th>\n",
       "      <th>target</th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>ts_hour</th>\n",
       "      <th>ts_minute</th>\n",
       "      <th>ts_weekday</th>\n",
       "      <th>ts_day</th>\n",
       "      <th>ts_month</th>\n",
       "      <th>ts_year</th>\n",
       "      <th>TE_brand_cat_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-12-01 07:51:27 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>1004781</td>\n",
       "      <td>huawei</td>\n",
       "      <td>247.27</td>\n",
       "      <td>569317987</td>\n",
       "      <td>3c378a32-dd69-4e1b-8251-2cfa0f831cd6</td>\n",
       "      <td>0</td>\n",
       "      <td>construction</td>\n",
       "      <td>tools</td>\n",
       "      <td>light</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 07:51:27</td>\n",
       "      <td>7</td>\n",
       "      <td>51</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.460027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-12-01 07:51:34 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>2401055</td>\n",
       "      <td>turbo</td>\n",
       "      <td>47.88</td>\n",
       "      <td>517451347</td>\n",
       "      <td>d3b2e38b-5d13-4b60-857c-f79f5674686b</td>\n",
       "      <td>0</td>\n",
       "      <td>appliances</td>\n",
       "      <td>kitchen</td>\n",
       "      <td>hood</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 07:51:34</td>\n",
       "      <td>7</td>\n",
       "      <td>51</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.251958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-12-01 07:51:36 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>1004856</td>\n",
       "      <td>samsung</td>\n",
       "      <td>124.10</td>\n",
       "      <td>580108461</td>\n",
       "      <td>f272b88b-0dcf-48b8-a466-7398dcda9d3b</td>\n",
       "      <td>0</td>\n",
       "      <td>construction</td>\n",
       "      <td>tools</td>\n",
       "      <td>light</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 07:51:36</td>\n",
       "      <td>7</td>\n",
       "      <td>51</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.481047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-12-01 07:51:36 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>1004751</td>\n",
       "      <td>samsung</td>\n",
       "      <td>192.77</td>\n",
       "      <td>545521992</td>\n",
       "      <td>686fc0f9-193e-4f81-95ec-02552cd596fe</td>\n",
       "      <td>0</td>\n",
       "      <td>construction</td>\n",
       "      <td>tools</td>\n",
       "      <td>light</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 07:51:36</td>\n",
       "      <td>7</td>\n",
       "      <td>51</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.481047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-12-01 07:51:37 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>1801906</td>\n",
       "      <td>tcl</td>\n",
       "      <td>231.13</td>\n",
       "      <td>552287591</td>\n",
       "      <td>681fbfd6-d352-4f3e-8ba6-5219bc0d3071</td>\n",
       "      <td>0</td>\n",
       "      <td>appliances</td>\n",
       "      <td>personal</td>\n",
       "      <td>massager</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 07:51:37</td>\n",
       "      <td>7</td>\n",
       "      <td>51</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.413226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461352</th>\n",
       "      <td>2019-11-30 19:56:57 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>1005174</td>\n",
       "      <td>samsung</td>\n",
       "      <td>591.75</td>\n",
       "      <td>515392975</td>\n",
       "      <td>0331b275-b924-4ff2-86e4-2239e4ce31b9</td>\n",
       "      <td>1</td>\n",
       "      <td>electronics</td>\n",
       "      <td>smartphone</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 19:56:57</td>\n",
       "      <td>19</td>\n",
       "      <td>56</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.403044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461353</th>\n",
       "      <td>2019-11-30 19:56:59 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>11400268</td>\n",
       "      <td>xiaomi</td>\n",
       "      <td>19.79</td>\n",
       "      <td>514447709</td>\n",
       "      <td>d42ae3ca-f27b-41db-9b43-d8cf99f2f637</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 19:56:59</td>\n",
       "      <td>19</td>\n",
       "      <td>56</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.290165</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461354</th>\n",
       "      <td>2019-11-30 19:57:10 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>3200090</td>\n",
       "      <td>kenwood</td>\n",
       "      <td>175.01</td>\n",
       "      <td>512602651</td>\n",
       "      <td>8cd3b00d-911b-4cb0-8b7d-e8712a791149</td>\n",
       "      <td>1</td>\n",
       "      <td>appliances</td>\n",
       "      <td>kitchen</td>\n",
       "      <td>meat_grinder</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 19:57:10</td>\n",
       "      <td>19</td>\n",
       "      <td>57</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.297872</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461355</th>\n",
       "      <td>2019-11-30 19:57:16 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>3600453</td>\n",
       "      <td>indesit</td>\n",
       "      <td>187.08</td>\n",
       "      <td>536074530</td>\n",
       "      <td>730ef938-d131-48ff-a815-3af631dcb5ea</td>\n",
       "      <td>1</td>\n",
       "      <td>appliances</td>\n",
       "      <td>kitchen</td>\n",
       "      <td>washer</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 19:57:16</td>\n",
       "      <td>19</td>\n",
       "      <td>57</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.344384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11461356</th>\n",
       "      <td>2019-11-30 19:57:21 UTC</td>\n",
       "      <td>purchase</td>\n",
       "      <td>5100816</td>\n",
       "      <td>xiaomi</td>\n",
       "      <td>32.15</td>\n",
       "      <td>556929237</td>\n",
       "      <td>3c6c445f-2755-4c93-b421-289de70c53d4</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-11-30 19:57:21</td>\n",
       "      <td>19</td>\n",
       "      <td>57</td>\n",
       "      <td>5</td>\n",
       "      <td>30</td>\n",
       "      <td>11</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.290165</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11461357 rows × 20 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                       event_time event_type  product_id    brand   price  \\\n",
       "0         2019-12-01 07:51:27 UTC       cart     1004781   huawei  247.27   \n",
       "1         2019-12-01 07:51:34 UTC       cart     2401055    turbo   47.88   \n",
       "2         2019-12-01 07:51:36 UTC       cart     1004856  samsung  124.10   \n",
       "3         2019-12-01 07:51:36 UTC       cart     1004751  samsung  192.77   \n",
       "4         2019-12-01 07:51:37 UTC       cart     1801906      tcl  231.13   \n",
       "...                           ...        ...         ...      ...     ...   \n",
       "11461352  2019-11-30 19:56:57 UTC   purchase     1005174  samsung  591.75   \n",
       "11461353  2019-11-30 19:56:59 UTC   purchase    11400268   xiaomi   19.79   \n",
       "11461354  2019-11-30 19:57:10 UTC   purchase     3200090  kenwood  175.01   \n",
       "11461355  2019-11-30 19:57:16 UTC   purchase     3600453  indesit  187.08   \n",
       "11461356  2019-11-30 19:57:21 UTC   purchase     5100816   xiaomi   32.15   \n",
       "\n",
       "            user_id                          user_session  target  \\\n",
       "0         569317987  3c378a32-dd69-4e1b-8251-2cfa0f831cd6       0   \n",
       "1         517451347  d3b2e38b-5d13-4b60-857c-f79f5674686b       0   \n",
       "2         580108461  f272b88b-0dcf-48b8-a466-7398dcda9d3b       0   \n",
       "3         545521992  686fc0f9-193e-4f81-95ec-02552cd596fe       0   \n",
       "4         552287591  681fbfd6-d352-4f3e-8ba6-5219bc0d3071       0   \n",
       "...             ...                                   ...     ...   \n",
       "11461352  515392975  0331b275-b924-4ff2-86e4-2239e4ce31b9       1   \n",
       "11461353  514447709  d42ae3ca-f27b-41db-9b43-d8cf99f2f637       1   \n",
       "11461354  512602651  8cd3b00d-911b-4cb0-8b7d-e8712a791149       1   \n",
       "11461355  536074530  730ef938-d131-48ff-a815-3af631dcb5ea       1   \n",
       "11461356  556929237  3c6c445f-2755-4c93-b421-289de70c53d4       1   \n",
       "\n",
       "                 cat_0       cat_1         cat_2 cat_3            timestamp  \\\n",
       "0         construction       tools         light  <NA>  2019-12-01 07:51:27   \n",
       "1           appliances     kitchen          hood  <NA>  2019-12-01 07:51:34   \n",
       "2         construction       tools         light  <NA>  2019-12-01 07:51:36   \n",
       "3         construction       tools         light  <NA>  2019-12-01 07:51:36   \n",
       "4           appliances    personal      massager  <NA>  2019-12-01 07:51:37   \n",
       "...                ...         ...           ...   ...                  ...   \n",
       "11461352   electronics  smartphone       UNKNOWN  <NA>  2019-11-30 19:56:57   \n",
       "11461353          <NA>        <NA>       UNKNOWN  <NA>  2019-11-30 19:56:59   \n",
       "11461354    appliances     kitchen  meat_grinder  <NA>  2019-11-30 19:57:10   \n",
       "11461355    appliances     kitchen        washer  <NA>  2019-11-30 19:57:16   \n",
       "11461356          <NA>        <NA>       UNKNOWN  <NA>  2019-11-30 19:57:21   \n",
       "\n",
       "          ts_hour  ts_minute  ts_weekday  ts_day  ts_month  ts_year  \\\n",
       "0               7         51           6       1        12     2019   \n",
       "1               7         51           6       1        12     2019   \n",
       "2               7         51           6       1        12     2019   \n",
       "3               7         51           6       1        12     2019   \n",
       "4               7         51           6       1        12     2019   \n",
       "...           ...        ...         ...     ...       ...      ...   \n",
       "11461352       19         56           5      30        11     2019   \n",
       "11461353       19         56           5      30        11     2019   \n",
       "11461354       19         57           5      30        11     2019   \n",
       "11461355       19         57           5      30        11     2019   \n",
       "11461356       19         57           5      30        11     2019   \n",
       "\n",
       "          TE_brand_cat_2  \n",
       "0               0.460027  \n",
       "1               0.251958  \n",
       "2               0.481047  \n",
       "3               0.481047  \n",
       "4               0.413226  \n",
       "...                  ...  \n",
       "11461352        0.403044  \n",
       "11461353        0.290165  \n",
       "11461354        0.297872  \n",
       "11461355        0.344384  \n",
       "11461356        0.290165  \n",
       "\n",
       "[11461357 rows x 20 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "te = te.reset_index()\n",
    "te.columns = ['brand', 'cat_2', 'TE_brand_cat_2']\n",
    "df_train.merge(te, how='left', left_on=['brand', 'cat_2'], right_on=['brand', 'cat_2'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*Target Encoding* creates a new features, which can be used by the model for training. The advantage of *Target Encoding* is, that it process the categorical features and makes them easier accessible to the model during training and validation.<br><br>\n",
    "Tree-based model requires to create a split for each categorical value (depending on the exact model). *Target Encoding* saves to create many splits for the model. In particular, when applying *Target Encoding* to multiple columns, it reduces significantly the number of splits. The model can directly operate on the probablities/averages and creates a split based on them.<br>\n",
    "Another advantage is, that some boosted-tree libraries, such as XGBoost, cannot handle categorical features. The library requires to hot-n encode them. Categorical features with large cardinality (e.g. >100) are inefficient to store as hot-n.<br><br>\n",
    "Deep learning models often apply Embedding Layers to categorical features. Embedding layer can overfit quickly and categorical values with low frequencies have ony a few gradient descent updates and can memorize the training data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Smoothing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The introduced *Target Encoding* is a good first step, but it lacks to generalize well and it will tend to overfit, as well. Let's take a look on *Target Encoding* with the observation count:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>brand</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>UNKNOWN</th>\n",
       "      <td>0.301577</td>\n",
       "      <td>946612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a-case</th>\n",
       "      <td>0.264910</td>\n",
       "      <td>2884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a-derma</th>\n",
       "      <td>0.150442</td>\n",
       "      <td>113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a-elita</th>\n",
       "      <td>0.275862</td>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>a-mega</th>\n",
       "      <td>0.340426</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zuru</th>\n",
       "      <td>0.285714</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zvezda</th>\n",
       "      <td>0.444444</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zwilling</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zwillingjahenckels</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>zyxel</th>\n",
       "      <td>0.285714</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4638 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      target        \n",
       "                        mean   count\n",
       "brand                               \n",
       "UNKNOWN             0.301577  946612\n",
       "a-case              0.264910    2884\n",
       "a-derma             0.150442     113\n",
       "a-elita             0.275862      29\n",
       "a-mega              0.340426      47\n",
       "...                      ...     ...\n",
       "zuru                0.285714      28\n",
       "zvezda              0.444444       9\n",
       "zwilling            0.000000       2\n",
       "zwillingjahenckels  0.000000      10\n",
       "zyxel               0.285714      14\n",
       "\n",
       "[4638 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train[[cat, 'target']].groupby(cat).agg(['mean', 'count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "dd = df_train[[cat, 'target']].groupby(cat).agg(['mean', 'count']).reset_index()['target']['count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.0, 50.0)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAARFElEQVR4nO3dbYxcV33H8e8PJzwIUJOQxXJtI6fgFgVUHLQNQfAiBAEhQXWQaBTUQoRSmUpBAok+OLzhQY0UpEJapDaqaVJMBQQLSGORtMUNkSgvSNgEk0cQBhzFlomXh0Ai1FQJ/77YY3lkG+/M7Kxn7fP9SKO599x7Z/577PnN2TP3zqaqkCT14VnTLkCSdOIY+pLUEUNfkjpi6EtSRwx9SerIadMuAODss8+uDRs2TLsMSTqp3HPPPT+tqplRjlkRob9hwwbm5uamXYYknVSSPDLqMU7vSFJHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSR1bEFblH2rD1tmO2773u0hNciSSdWhzpS1JHDH1J6oihL0kdWTT0kzw3yd1JvpvkwSQfbe2fSfLjJLvbbVNrT5JPJdmT5L4kr17mn0GSNKRhPsh9Crioqp5McjrwzST/0bb9VVV96Yj93wpsbLfXADe0e0nSlC060q8FT7bV09utjnPIZuCz7bhvAWckWbP0UiVJSzXUnH6SVUl2AweBXVV1V9t0bZvCuT7Jc1rbWuDRgcP3tbYjH3NLkrkkc/Pz8+P/BJKkoQ0V+lX1TFVtAtYB5yd5JXAN8HLgj4CzgL8Z5YmraltVzVbV7MzMSH/tS5I0ppHO3qmqx4E7gYur6kCbwnkK+Ffg/LbbfmD9wGHrWpskacqGOXtnJskZbfl5wJuA7x2ap08S4DLggXbITuDd7SyeC4BfVtWBZahdkjSiYc7eWQNsT7KKhTeJHVX11SRfTzIDBNgN/EXb/3bgEmAP8GvgPROvWpI0lkVDv6ruA847RvtFv2X/Aq5eemmSpEnzilxJ6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4uGfpLnJrk7yXeTPJjko639nCR3JdmT5ItJnt3an9PW97TtG5b5Z5AkDWmYkf5TwEVV9SpgE3BxkguAjwPXV9XLgF8AV7X9rwJ+0dqvb/tJklaARUO/FjzZVk9vtwIuAr7U2rcDl7XlzW2dtv2NSTKpgiVJ4xtqTj/JqiS7gYPALuCHwONV9XTbZR+wti2vBR4FaNt/CbxogjVLksY0VOhX1TNVtQlYB5wPvHypT5xkS5K5JHPz8/NLfThJ0hBGOnunqh4H7gReC5yR5LS2aR2wvy3vB9YDtO2/A/zsGI+1rapmq2p2ZmZmvOolSSMZ5uydmSRntOXnAW8CHmYh/N/RdrsSuLUt72zrtO1fr6qaYM2SpDGdtvgurAG2J1nFwpvEjqr6apKHgJuT/C3wHeDGtv+NwL8l2QP8HLhiGeqWJI1h0dCvqvuA847R/iMW5vePbP9f4E8mUp0kaaK8IleSOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR1ZNPSTrE9yZ5KHkjyY5P2t/SNJ9ifZ3W6XDBxzTZI9Sb6f5C3L+QNIkoZ32hD7PA18sKruTfJC4J4ku9q266vq7wZ3TnIucAXwCuB3gf9O8vtV9cwkC5ckjW7RkX5VHaiqe9vyE8DDwNrjHLIZuLmqnqqqHwN7gPMnUawkaWlGmtNPsgE4D7irNb0vyX1JbkpyZmtbCzw6cNg+jvEmkWRLkrkkc/Pz86NXLkka2dChn+QFwJeBD1TVr4AbgJcCm4ADwCdGeeKq2lZVs1U1OzMzM8qhkqQxDRX6SU5nIfA/V1VfAaiqx6rqmar6DfBpDk/h7AfWDxy+rrVJkqZsmLN3AtwIPFxVnxxoXzOw29uBB9ryTuCKJM9Jcg6wEbh7ciVLksY1zNk7rwPeBdyfZHdr+xDwziSbgAL2Au8FqKoHk+wAHmLhzJ+rJ3nmzoattx3Vtve6Syf18JJ0Sls09Kvqm0COsen24xxzLXDtEuqSJC0Dr8iVpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHFg39JOuT3JnkoSQPJnl/az8rya4kP2j3Z7b2JPlUkj1J7kvy6uX+ISRJwxlmpP808MGqOhe4ALg6ybnAVuCOqtoI3NHWAd4KbGy3LcANE69akjSWRUO/qg5U1b1t+QngYWAtsBnY3nbbDlzWljcDn60F3wLOSLJm0oVLkkY30px+kg3AecBdwOqqOtA2/QRY3ZbXAo8OHLavtR35WFuSzCWZm5+fH7VuSdIYhg79JC8Avgx8oKp+NbitqgqoUZ64qrZV1WxVzc7MzIxyqCRpTEOFfpLTWQj8z1XVV1rzY4embdr9wda+H1g/cPi61iZJmrJhzt4JcCPwcFV9cmDTTuDKtnwlcOtA+7vbWTwXAL8cmAaSJE3RaUPs8zrgXcD9SXa3tg8B1wE7klwFPAJc3rbdDlwC7AF+DbxnkgVLksa3aOhX1TeB/JbNbzzG/gVcvcS6xrJh621Hte297tIpVCJJK5NX5EpSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdWTT0k9yU5GCSBwbaPpJkf5Ld7XbJwLZrkuxJ8v0kb1muwiVJoxtmpP8Z4OJjtF9fVZva7XaAJOcCVwCvaMf8U5JVkypWkrQ0i4Z+VX0D+PmQj7cZuLmqnqqqHwN7gPOXUJ8kaYKWMqf/viT3temfM1vbWuDRgX32tbajJNmSZC7J3Pz8/BLKkCQNa9zQvwF4KbAJOAB8YtQHqKptVTVbVbMzMzNjliFJGsVYoV9Vj1XVM1X1G+DTHJ7C2Q+sH9h1XWuTJK0AY4V+kjUDq28HDp3ZsxO4IslzkpwDbATuXlqJkqRJOW2xHZJ8AbgQODvJPuDDwIVJNgEF7AXeC1BVDybZATwEPA1cXVXPLEvlkqSRLRr6VfXOYzTfeJz9rwWuXUpRkqTl4RW5ktQRQ1+SOrLo9M6pbsPW245q23vdpVOoRJKWnyN9SepIFyP9Y43mwRG9pP440pekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUkS6+hmFcfhmbpFONI31J6oihL0kdMfQlqSOGviR1ZNHQT3JTkoNJHhhoOyvJriQ/aPdntvYk+VSSPUnuS/Lq5SxekjSaYUb6nwEuPqJtK3BHVW0E7mjrAG8FNrbbFuCGyZQpSZqERUO/qr4B/PyI5s3A9ra8HbhsoP2zteBbwBlJ1kyoVknSEo07p7+6qg605Z8Aq9vyWuDRgf32tbajJNmSZC7J3Pz8/JhlSJJGseQPcquqgBrjuG1VNVtVszMzM0stQ5I0hHGvyH0syZqqOtCmbw629v3A+oH91rW2U4p/aF3SyWrckf5O4Mq2fCVw60D7u9tZPBcAvxyYBpIkTdmiI/0kXwAuBM5Osg/4MHAdsCPJVcAjwOVt99uBS4A9wK+B9yxDzZKkMS0a+lX1zt+y6Y3H2LeAq5dalCRpeXhFriR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIfyN3wvy7upJWMkf6ktQRR/orgN/lI+lEcaQvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOuIpmycxLwSTNCpH+pLUEUf6J5Ajc0nTZuivcF6tK2mSlhT6SfYCTwDPAE9X1WySs4AvAhuAvcDlVfWLpZWpUflbhaRjmcSc/huqalNVzbb1rcAdVbURuKOtS5JWgOX4IHczsL0tbwcuW4bnkCSNYalz+gV8LUkB/1xV24DVVXWgbf8JsPpYBybZAmwBeMlLXrLEMjQJTglJp76lhv7rq2p/khcDu5J8b3BjVVV7QzhKe4PYBjA7O3vMfXTq8g1Gmo4lhX5V7W/3B5PcApwPPJZkTVUdSLIGODiBOjUhng0k9W3s0E/yfOBZVfVEW34z8DFgJ3AlcF27v3USherk42heWnmWMtJfDdyS5NDjfL6q/jPJt4EdSa4CHgEuX3qZmjYDXDo1jB36VfUj4FXHaP8Z8MalFCVJWh5ekatTgr+JSMPxC9ckqSOO9LUkng0knVwc6UtSRwx9SeqI0ztaUZZjusgPeaXDDH2dNAzvw+wLjcvQV7eO91uFoXpq8N/xaIa+pOM6XnAaqicfQ18a0aSDzuDUiWToSxMy6Q+hV/rj6eRk6EuaOD8vWbkMfekUcyqGqr+lTI6hL50AKz2IDdXpmMb/C0Nf0ooxTgiu9DfUcS3Xz2XoS9IyWmm/RRn6kjSklXDNwm97ExmWoS+pO6OeXXRo26nAb9mUpI4Y+pLUkWUL/SQXJ/l+kj1Jti7X80iShrcsoZ9kFfCPwFuBc4F3Jjl3OZ5LkjS85Rrpnw/sqaofVdX/ATcDm5fpuSRJQ0pVTf5Bk3cAF1fVn7f1dwGvqar3DeyzBdjSVl8JPDDxQk5OZwM/nXYRK4R9cZh9cZh9cdgfVNULRzlgaqdsVtU2YBtAkrmqmp1WLSuJfXGYfXGYfXGYfXFYkrlRj1mu6Z39wPqB9XWtTZI0RcsV+t8GNiY5J8mzgSuAncv0XJKkIS3L9E5VPZ3kfcB/AauAm6rqweMcsm056jhJ2ReH2ReH2ReH2ReHjdwXy/JBriRpZfKKXEnqiKEvSR2Zeuj3/HUNSW5KcjDJAwNtZyXZleQH7f7MadZ4oiRZn+TOJA8leTDJ+1t7d/2R5LlJ7k7y3dYXH23t5yS5q71WvthOkjjlJVmV5DtJvtrWe+2HvUnuT7L70Kma47w+phr6fl0DnwEuPqJtK3BHVW0E7mjrPXga+GBVnQtcAFzd/i/02B9PARdV1auATcDFSS4APg5cX1UvA34BXDW9Ek+o9wMPD6z32g8Ab6iqTQPXKYz8+pj2SL/rr2uoqm8APz+ieTOwvS1vBy47kTVNS1UdqKp72/ITLLzI19Jhf9SCJ9vq6e1WwEXAl1p7F32RZB1wKfAvbT102A/HMfLrY9qhvxZ4dGB9X2vr2eqqOtCWfwKsnmYx05BkA3AecBed9keb0tgNHAR2AT8EHq+qp9suvbxW/h74a+A3bf1F9NkPsPDG/7Uk97SvsYExXh/+5awVrKoqSVfn1CZ5AfBl4ANV9auFgd2Cnvqjqp4BNiU5A7gFePl0KzrxkrwNOFhV9yS5cMrlrASvr6r9SV4M7EryvcGNw74+pj3S9+sajvZYkjUA7f7glOs5YZKczkLgf66qvtKau+0PgKp6HLgTeC1wRpJDA7UeXiuvA/44yV4Wpn4vAv6B/voBgKra3+4PsjAQOJ8xXh/TDn2/ruFoO4Er2/KVwK1TrOWEaXO1NwIPV9UnBzZ11x9JZtoInyTPA97EwmccdwLvaLud8n1RVddU1bqq2sBCNny9qv6UzvoBIMnzk7zw0DLwZha+mXjk18fUr8hNcgkL83aHvq7h2qkWdAIl+QJwIQtfFfsY8GHg34EdwEuAR4DLq+rID3tPOUleD/wPcD+H528/xMK8flf9keQPWfhQbhULA7MdVfWxJL/Hwoj3LOA7wJ9V1VPTq/TEadM7f1lVb+uxH9rPfEtbPQ34fFVdm+RFjPj6mHroS5JOnGlP70iSTiBDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXk/wHrJj1Fj591awAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.bar(dd.groupby('count').count().index.to_array(), dd.groupby('count').count().to_array())\n",
    "plt.xlim(0,50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can observe, that the observation count for some categories are 1. This means, that we have only one data point to calculate the average and *Target Encoding* overfits to these values. Therefore, we need to adjust the calculation:<br><li>if the number of observation is <b>high</b>, we want to use the <b>mean of this category value</b><br><li>if the number of observation is <b>low</b>, we want to use the <b>global mean</b>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\\begin{equation} \\label{eq:te}\n",
    "TE_{target}([Categories]) = \\frac{count([Categories]) * mean_{target}([Categories]) + w_{smoothing} * mean_{target}(global)}{count([Categories]) + w_{smoothing}}\n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A simple way is to calculate a weighted average of the category value mean ($mean_{target}[Categories]$) and the global mean ($mean_{target}(global)$).\n",
    "\n",
    "We add a smoothing weight $w_{smoothing} \\in \\mathbb{N}$. A bigger $w_{smoothing}$ relates to that *Target Encoding* is closer to the global mean.  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Practice"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, it is your turn. Let's try to implement *Target Encoding* as a function.<br><br>\n",
    "\n",
    "**ToDo**:\n",
    "<li>We use a smoothing factor of w=20<br>\n",
    "<li>We Target Encode the columns feat=['brand', 'cat_2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "### ToDo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "############### Solution ###############\n",
    "feat = ['brand', 'cat_2']\n",
    "w = 20\n",
    "\n",
    "mean_global = df_train.target.mean()\n",
    "te = df_train.groupby(feat)['target'].agg(['mean','count']).reset_index()\n",
    "te['TE_brand_cat_2'] = ((te['mean']*te['count'])+(mean_global*w))/(te['count']+w)\n",
    "\n",
    "df_train = df_train.merge(te, on=feat, how='left')\n",
    "df_valid = df_valid.merge( te, on=feat, how='left' )\n",
    "df_test = df_test.merge( te, on=feat, how='left' )\n",
    "df_valid['TE_brand_cat_2'] = df_valid['TE_brand_cat_2'].fillna(mean_global)\n",
    "df_test['TE_brand_cat_2'] = df_test['TE_brand_cat_2'].fillna(mean_global)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "############### Solution End ###########"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Showing the effect of smoothing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A tree-based or deep learning based model cannot easily capture the idea of smoothing. We show the positive effect of smoothing on the target. Therefore, we compare *Target Encoding* with and without smoothing."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TargetEncoding without smoothing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat = ['ts_weekday', 'ts_hour', 'cat_2', 'brand']\n",
    "te = df_train.groupby(cat).target.agg(['mean', 'count']).reset_index()\n",
    "te.columns = cat + ['TE_mean', 'TE_count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid = df_valid.merge(te, on=cat, how='left')\n",
    "df_valid['error'] = (df_valid['target'] - (df_valid['TE_mean']>=0.5)).abs()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_global = df_train.target.mean()\n",
    "df_valid['TE_mean'] = df_valid['TE_mean'].fillna(mean_global)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TargetEncoding with smoothing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = 20\n",
    "df_valid['TE_mean_smoothed'] = ((df_valid['TE_mean']*df_valid['TE_count'])+(mean_global*w))/(df_valid['TE_count']+w)\n",
    "df_valid['TE_mean_smoothed'] = df_valid['TE_mean_smoothed'].fillna(mean_global)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_valid['error_smoothed'] = (df_valid['target'] - (df_valid['TE_mean_smoothed']>=0.5)).abs()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's look at the error based on the number of observations. We can see, that the categorical values with low observation count (1, 2, 3) have a lower error rate with smoothing than without smoothing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TE_count\n",
       "1        0.433183\n",
       "2        0.487893\n",
       "3        0.414957\n",
       "4        0.461962\n",
       "5        0.418925\n",
       "           ...   \n",
       "13672    0.477014\n",
       "13789    0.516581\n",
       "13806    0.484599\n",
       "13847    0.469956\n",
       "15033    0.328155\n",
       "Name: error, Length: 2068, dtype: float64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid[['TE_count', 'error']].groupby('TE_count').error.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TE_count\n",
       "1        0.330565\n",
       "2        0.337878\n",
       "3        0.340056\n",
       "4        0.336385\n",
       "5        0.344421\n",
       "           ...   \n",
       "13672    0.477014\n",
       "13789    0.516581\n",
       "13806    0.484599\n",
       "13847    0.469956\n",
       "15033    0.328155\n",
       "Name: error_smoothed, Length: 2068, dtype: float64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid[['TE_count', 'error_smoothed']].groupby('TE_count').error_smoothed.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can look at the roc_auc values as well:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.57453584823663"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(df_valid['target'].to_pandas().astype(int).values, \n",
    "              df_valid['TE_mean'].to_pandas().values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5829179874937375"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(df_valid['target'].to_pandas().astype(int).values, \n",
    "              df_valid['TE_mean_smoothed'].to_pandas().values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Improve TargetEncoding with out-of-fold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can still improve our *Target Encoding* function. We can even make it more generalizable, if we apply an *out of fold calculation*. <br><br>In our current definition, we use the full training dataset to *Target Encode* the training dataset and validation/test dataset. Therefore, we will likely overfit slightly on our training dataset, because we use the information from it to encode the categorical values. A better strategy is to use *out of fold*:\n",
    "<li> use the full training dataset to encode the validation/test dataset<br>\n",
    "<li> split the training dataset in k-folds and encode the i-th fold by using all folds except of the i-th one<br><br>\n",
    "The following figure visualize the strategy for k=5:"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src='../images/te_oof.png' width=50%>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The k-fold can be generated by a random split or by a timestamp depending on the dataset."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We restart the session."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'status': 'ok', 'restart': True}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "app = IPython.Application.instance()\n",
    "app.kernel.do_shutdown(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import IPython\n",
    "\n",
    "import pandas as pd\n",
    "import cudf\n",
    "import numpy as np\n",
    "import cupy\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "df_train = cudf.read_parquet('../data/train.parquet')\n",
    "df_valid = cudf.read_parquet('../data/valid.parquet')\n",
    "df_test = cudf.read_parquet('../data/test.parquet')\n",
    "\n",
    "df_train['brand'] = df_train['brand'].fillna('UNKNOWN')\n",
    "df_valid['brand'] = df_valid['brand'].fillna('UNKNOWN')\n",
    "df_test['brand'] = df_test['brand'].fillna('UNKNOWN')\n",
    "df_train['cat_2'] = df_train['cat_2'].fillna('UNKNOWN')\n",
    "df_valid['cat_2'] = df_valid['cat_2'].fillna('UNKNOWN')\n",
    "df_test['cat_2'] = df_test['cat_2'].fillna('UNKNOWN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def target_encode(train, valid, col, target, kfold=5, smooth=20):\n",
    "    \"\"\"\n",
    "        train:  train dataset\n",
    "        valid:  validation dataset\n",
    "        col:   column which will be encoded (in the example RESOURCE)\n",
    "        target: target column which will be used to calculate the statistic\n",
    "    \"\"\"\n",
    "    \n",
    "    # We assume that the train dataset is shuffled\n",
    "    train['kfold'] = ((train.index) % kfold)\n",
    "    # We keep the original order as cudf merge will not preserve the original order\n",
    "    train['org_sorting'] = cupy.arange(len(train), dtype=\"int32\")\n",
    "    # We create the output column, we fill with 0\n",
    "    col_name = '_'.join(col)\n",
    "    train['TE_' + col_name] = 0.\n",
    "    for i in range(kfold):\n",
    "        ###################################\n",
    "        # filter for out of fold\n",
    "        # calculate the mean/counts per group category\n",
    "        # calculate the global mean for the oof\n",
    "        # calculate the smoothed TE\n",
    "        # merge it to the original dataframe\n",
    "        ###################################\n",
    "        \n",
    "        df_tmp = train[train['kfold']!=i]\n",
    "        mn = df_tmp[target].mean()\n",
    "        df_tmp = df_tmp[col + [target]].groupby(col).agg(['mean', 'count']).reset_index()\n",
    "        df_tmp.columns = col + ['mean', 'count']\n",
    "        df_tmp['TE_tmp'] = ((df_tmp['mean']*df_tmp['count'])+(mn*smooth)) / (df_tmp['count']+smooth)\n",
    "        df_tmp_m = train[col + ['kfold', 'org_sorting', 'TE_' + col_name]].merge(df_tmp, how='left', left_on=col, right_on=col).sort_values('org_sorting')\n",
    "        df_tmp_m.loc[df_tmp_m['kfold']==i, 'TE_' + col_name] = df_tmp_m.loc[df_tmp_m['kfold']==i, 'TE_tmp']\n",
    "        train['TE_' + col_name] = df_tmp_m['TE_' + col_name].fillna(mn).values\n",
    "\n",
    "    \n",
    "    ###################################\n",
    "    # calculate the mean/counts per group for the full training dataset\n",
    "    # calculate the global mean\n",
    "    # calculate the smoothed TE\n",
    "    # merge it to the original dataframe\n",
    "    # drop all temp columns\n",
    "    ###################################    \n",
    "    \n",
    "    df_tmp = train[col + [target]].groupby(col).agg(['mean', 'count']).reset_index()\n",
    "    mn = train[target].mean()\n",
    "    df_tmp.columns = col + ['mean', 'count']\n",
    "    df_tmp['TE_tmp'] = ((df_tmp['mean']*df_tmp['count'])+(mn*smooth)) / (df_tmp['count']+smooth)\n",
    "    valid['org_sorting'] = cupy.arange(len(valid), dtype=\"int32\")\n",
    "    df_tmp_m = valid[col + ['org_sorting']].merge(df_tmp, how='left', left_on=col, right_on=col).sort_values('org_sorting')\n",
    "    valid['TE_' + col_name] = df_tmp_m['TE_tmp'].fillna(mn).values\n",
    "    \n",
    "    valid = valid.drop('org_sorting', axis=1)\n",
    "    train = train.drop('kfold', axis=1)\n",
    "    train = train.drop('org_sorting', axis=1)\n",
    "    return(train, valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.64 s, sys: 4.16 s, total: 7.8 s\n",
      "Wall time: 7.56 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df_train, df_valid = target_encode(df_train, df_valid, ['ts_weekday', 'ts_hour', 'cat_2', 'brand'], 'target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event_time</th>\n",
       "      <th>event_type</th>\n",
       "      <th>product_id</th>\n",
       "      <th>brand</th>\n",
       "      <th>price</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_session</th>\n",
       "      <th>target</th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>ts_hour</th>\n",
       "      <th>ts_minute</th>\n",
       "      <th>ts_weekday</th>\n",
       "      <th>ts_day</th>\n",
       "      <th>ts_month</th>\n",
       "      <th>ts_year</th>\n",
       "      <th>TE_ts_weekday_ts_hour_cat_2_brand</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2019-12-01 00:00:28 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>17800342</td>\n",
       "      <td>zeta</td>\n",
       "      <td>66.90</td>\n",
       "      <td>550465671</td>\n",
       "      <td>22650a62-2d9c-4151-9f41-2674ec6d32d5</td>\n",
       "      <td>0</td>\n",
       "      <td>computers</td>\n",
       "      <td>desktop</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 00:00:28</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.301241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2019-12-01 00:00:39 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>3701309</td>\n",
       "      <td>polaris</td>\n",
       "      <td>89.32</td>\n",
       "      <td>543733099</td>\n",
       "      <td>a65116f4-ac53-4a41-ad68-6606788e674c</td>\n",
       "      <td>0</td>\n",
       "      <td>appliances</td>\n",
       "      <td>environment</td>\n",
       "      <td>vacuum</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 00:00:39</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.333539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2019-12-01 00:00:40 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>3701309</td>\n",
       "      <td>polaris</td>\n",
       "      <td>89.32</td>\n",
       "      <td>543733099</td>\n",
       "      <td>a65116f4-ac53-4a41-ad68-6606788e674c</td>\n",
       "      <td>0</td>\n",
       "      <td>appliances</td>\n",
       "      <td>environment</td>\n",
       "      <td>vacuum</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 00:00:40</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.319065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2019-12-01 00:00:41 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>3701309</td>\n",
       "      <td>polaris</td>\n",
       "      <td>89.32</td>\n",
       "      <td>543733099</td>\n",
       "      <td>a65116f4-ac53-4a41-ad68-6606788e674c</td>\n",
       "      <td>0</td>\n",
       "      <td>appliances</td>\n",
       "      <td>environment</td>\n",
       "      <td>vacuum</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 00:00:41</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.333539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2019-12-01 00:01:56 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>1004767</td>\n",
       "      <td>samsung</td>\n",
       "      <td>235.60</td>\n",
       "      <td>579970209</td>\n",
       "      <td>c6946211-ce70-4228-95ce-fd7fccdde63c</td>\n",
       "      <td>0</td>\n",
       "      <td>construction</td>\n",
       "      <td>tools</td>\n",
       "      <td>light</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2019-12-01 00:01:56</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>2019</td>\n",
       "      <td>0.466269</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                event_time event_type  product_id    brand   price    user_id  \\\n",
       "0  2019-12-01 00:00:28 UTC       cart    17800342     zeta   66.90  550465671   \n",
       "1  2019-12-01 00:00:39 UTC       cart     3701309  polaris   89.32  543733099   \n",
       "2  2019-12-01 00:00:40 UTC       cart     3701309  polaris   89.32  543733099   \n",
       "3  2019-12-01 00:00:41 UTC       cart     3701309  polaris   89.32  543733099   \n",
       "4  2019-12-01 00:01:56 UTC       cart     1004767  samsung  235.60  579970209   \n",
       "\n",
       "                           user_session  target         cat_0        cat_1  \\\n",
       "0  22650a62-2d9c-4151-9f41-2674ec6d32d5       0     computers      desktop   \n",
       "1  a65116f4-ac53-4a41-ad68-6606788e674c       0    appliances  environment   \n",
       "2  a65116f4-ac53-4a41-ad68-6606788e674c       0    appliances  environment   \n",
       "3  a65116f4-ac53-4a41-ad68-6606788e674c       0    appliances  environment   \n",
       "4  c6946211-ce70-4228-95ce-fd7fccdde63c       0  construction        tools   \n",
       "\n",
       "     cat_2 cat_3            timestamp  ts_hour  ts_minute  ts_weekday  ts_day  \\\n",
       "0  UNKNOWN  <NA>  2019-12-01 00:00:28        0          0           6       1   \n",
       "1   vacuum  <NA>  2019-12-01 00:00:39        0          0           6       1   \n",
       "2   vacuum  <NA>  2019-12-01 00:00:40        0          0           6       1   \n",
       "3   vacuum  <NA>  2019-12-01 00:00:41        0          0           6       1   \n",
       "4    light  <NA>  2019-12-01 00:01:56        0          1           6       1   \n",
       "\n",
       "   ts_month  ts_year  TE_ts_weekday_ts_hour_cat_2_brand  \n",
       "0        12     2019                           0.301241  \n",
       "1        12     2019                           0.333539  \n",
       "2        12     2019                           0.319065  \n",
       "3        12     2019                           0.333539  \n",
       "4        12     2019                           0.466269  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event_time</th>\n",
       "      <th>event_type</th>\n",
       "      <th>product_id</th>\n",
       "      <th>brand</th>\n",
       "      <th>price</th>\n",
       "      <th>user_id</th>\n",
       "      <th>user_session</th>\n",
       "      <th>target</th>\n",
       "      <th>cat_0</th>\n",
       "      <th>cat_1</th>\n",
       "      <th>cat_2</th>\n",
       "      <th>cat_3</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>ts_hour</th>\n",
       "      <th>ts_minute</th>\n",
       "      <th>ts_weekday</th>\n",
       "      <th>ts_day</th>\n",
       "      <th>ts_month</th>\n",
       "      <th>ts_year</th>\n",
       "      <th>TE_ts_weekday_ts_hour_cat_2_brand</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2020-03-01 00:00:59 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>6902464</td>\n",
       "      <td>zlatek</td>\n",
       "      <td>49.91</td>\n",
       "      <td>531574188</td>\n",
       "      <td>48714293-b3f9-4946-8135-eb1ea05ead74</td>\n",
       "      <td>0</td>\n",
       "      <td>electronics</td>\n",
       "      <td>telephone</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2020-03-01 00:00:59</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2020</td>\n",
       "      <td>0.366924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2020-03-01 00:01:20 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>1002544</td>\n",
       "      <td>apple</td>\n",
       "      <td>397.10</td>\n",
       "      <td>622090790</td>\n",
       "      <td>fb5b918c-f1f6-48d9-bcf4-7eb46e83fc6b</td>\n",
       "      <td>0</td>\n",
       "      <td>construction</td>\n",
       "      <td>tools</td>\n",
       "      <td>light</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2020-03-01 00:01:20</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2020</td>\n",
       "      <td>0.472616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2020-03-01 00:01:52 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>1003316</td>\n",
       "      <td>apple</td>\n",
       "      <td>823.70</td>\n",
       "      <td>622090543</td>\n",
       "      <td>b821ee79-96fe-4979-be9d-21ee2e6777c3</td>\n",
       "      <td>0</td>\n",
       "      <td>construction</td>\n",
       "      <td>tools</td>\n",
       "      <td>light</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2020-03-01 00:01:52</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2020</td>\n",
       "      <td>0.472616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2020-03-01 00:02:14 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>16600067</td>\n",
       "      <td>rivertoys</td>\n",
       "      <td>422.15</td>\n",
       "      <td>616437533</td>\n",
       "      <td>aad023bc-c858-47ab-a3a7-ff4654f11b9a</td>\n",
       "      <td>0</td>\n",
       "      <td>sport</td>\n",
       "      <td>trainer</td>\n",
       "      <td>UNKNOWN</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2020-03-01 00:02:14</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2020</td>\n",
       "      <td>0.333567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2020-03-01 00:02:15 UTC</td>\n",
       "      <td>cart</td>\n",
       "      <td>3701428</td>\n",
       "      <td>arnica</td>\n",
       "      <td>69.24</td>\n",
       "      <td>516454226</td>\n",
       "      <td>ee22b80c-ed3e-3c83-d397-fb69a44d4864</td>\n",
       "      <td>0</td>\n",
       "      <td>appliances</td>\n",
       "      <td>environment</td>\n",
       "      <td>vacuum</td>\n",
       "      <td>&lt;NA&gt;</td>\n",
       "      <td>2020-03-01 00:02:15</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2020</td>\n",
       "      <td>0.379022</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                event_time event_type  product_id      brand   price  \\\n",
       "0  2020-03-01 00:00:59 UTC       cart     6902464     zlatek   49.91   \n",
       "1  2020-03-01 00:01:20 UTC       cart     1002544      apple  397.10   \n",
       "2  2020-03-01 00:01:52 UTC       cart     1003316      apple  823.70   \n",
       "3  2020-03-01 00:02:14 UTC       cart    16600067  rivertoys  422.15   \n",
       "4  2020-03-01 00:02:15 UTC       cart     3701428     arnica   69.24   \n",
       "\n",
       "     user_id                          user_session  target         cat_0  \\\n",
       "0  531574188  48714293-b3f9-4946-8135-eb1ea05ead74       0   electronics   \n",
       "1  622090790  fb5b918c-f1f6-48d9-bcf4-7eb46e83fc6b       0  construction   \n",
       "2  622090543  b821ee79-96fe-4979-be9d-21ee2e6777c3       0  construction   \n",
       "3  616437533  aad023bc-c858-47ab-a3a7-ff4654f11b9a       0         sport   \n",
       "4  516454226  ee22b80c-ed3e-3c83-d397-fb69a44d4864       0    appliances   \n",
       "\n",
       "         cat_1    cat_2 cat_3            timestamp  ts_hour  ts_minute  \\\n",
       "0    telephone  UNKNOWN  <NA>  2020-03-01 00:00:59        0          0   \n",
       "1        tools    light  <NA>  2020-03-01 00:01:20        0          1   \n",
       "2        tools    light  <NA>  2020-03-01 00:01:52        0          1   \n",
       "3      trainer  UNKNOWN  <NA>  2020-03-01 00:02:14        0          2   \n",
       "4  environment   vacuum  <NA>  2020-03-01 00:02:15        0          2   \n",
       "\n",
       "   ts_weekday  ts_day  ts_month  ts_year  TE_ts_weekday_ts_hour_cat_2_brand  \n",
       "0           6       1         3     2020                           0.366924  \n",
       "1           6       1         3     2020                           0.472616  \n",
       "2           6       1         3     2020                           0.472616  \n",
       "3           6       1         3     2020                           0.333567  \n",
       "4           6       1         3     2020                           0.379022  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_valid.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<li> Target Encoding calculates statistics of a target column given one or more categorical features<br>\n",
    "<li> Target Encoding smooths the statistics as a weighted average of the category value and the global statistic<br>\n",
    "<li> Target Encoding uses a out-of-fold strategy to prevent overfitting to the training dataset.<br><br>\n",
    "    \n",
    "We can see the advantage of using *Target Encoding* as a feature engineering step. A tree-based model or a neural network learns the average probability for the category value. However, neither model is designed to prevent overfitting. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's compare the runtime between pandas and cuDF. The implementation depends only on the DataFrame object (calling function of the object) and does not require any pd / cuDF function. Therefore, we can use the same implementation and just use pandas.DataFrame and cuDF.DataFrame. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We restart the session."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'status': 'ok', 'restart': True}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "app = IPython.Application.instance()\n",
    "app.kernel.do_shutdown(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import IPython\n",
    "\n",
    "import pandas as pd\n",
    "import cudf\n",
    "import numpy as np\n",
    "import cupy\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "df_train = cudf.read_parquet('../data/train.parquet')\n",
    "df_valid = cudf.read_parquet('../data/valid.parquet')\n",
    "\n",
    "df_train['brand'] = df_train['brand'].fillna('UNKNOWN')\n",
    "df_valid['brand'] = df_valid['brand'].fillna('UNKNOWN')\n",
    "df_train['cat_2'] = df_train['cat_2'].fillna('UNKNOWN')\n",
    "df_valid['cat_2'] = df_valid['cat_2'].fillna('UNKNOWN')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def target_encode(train, valid, col, target, kfold=5, smooth=20, gpu=True):\n",
    "    \"\"\"\n",
    "        train:  train dataset\n",
    "        valid:  validation dataset\n",
    "        col:   column which will be encoded (in the example RESOURCE)\n",
    "        target: target column which will be used to calculate the statistic\n",
    "    \"\"\"\n",
    "    \n",
    "    # We assume that the train dataset is shuffled\n",
    "    train['kfold'] = ((train.index) % kfold)\n",
    "    # We keep the original order as cudf merge will not preserve the original order\n",
    "    if gpu:\n",
    "        train['org_sorting'] = cupy.arange(len(train), dtype=\"int32\")\n",
    "    else:\n",
    "        train['org_sorting'] = np.arange(len(train), dtype=\"int32\")\n",
    "    # We create the output column, we fill with 0\n",
    "    col_name = '_'.join(col)\n",
    "    train['TE_' + col_name] = 0.\n",
    "    for i in range(kfold):\n",
    "        ###################################\n",
    "        # filter for out of fold\n",
    "        # calculate the mean/counts per group category\n",
    "        # calculate the global mean for the oof\n",
    "        # calculate the smoothed TE\n",
    "        # merge it to the original dataframe\n",
    "        ###################################\n",
    "        \n",
    "        df_tmp = train[train['kfold']!=i]\n",
    "        mn = df_tmp[target].mean()\n",
    "        df_tmp = df_tmp[col + [target]].groupby(col).agg(['mean', 'count']).reset_index()\n",
    "        df_tmp.columns = col + ['mean', 'count']\n",
    "        df_tmp['TE_tmp'] = ((df_tmp['mean']*df_tmp['count'])+(mn*smooth)) / (df_tmp['count']+smooth)\n",
    "        df_tmp_m = train[col + ['kfold', 'org_sorting', 'TE_' + col_name]].merge(df_tmp, how='left', left_on=col, right_on=col).sort_values('org_sorting')\n",
    "        df_tmp_m.loc[df_tmp_m['kfold']==i, 'TE_' + col_name] = df_tmp_m.loc[df_tmp_m['kfold']==i, 'TE_tmp']\n",
    "        train['TE_' + col_name] = df_tmp_m['TE_' + col_name].fillna(mn).values\n",
    "\n",
    "    \n",
    "    ###################################\n",
    "    # calculate the mean/counts per group for the full training dataset\n",
    "    # calculate the global mean\n",
    "    # calculate the smoothed TE\n",
    "    # merge it to the original dataframe\n",
    "    # drop all temp columns\n",
    "    ###################################    \n",
    "    \n",
    "    df_tmp = train[col + [target]].groupby(col).agg(['mean', 'count']).reset_index()\n",
    "    mn = train[target].mean()\n",
    "    df_tmp.columns = col + ['mean', 'count']\n",
    "    df_tmp['TE_tmp'] = ((df_tmp['mean']*df_tmp['count'])+(mn*smooth)) / (df_tmp['count']+smooth)\n",
    "    if gpu:\n",
    "        valid['org_sorting'] = cupy.arange(len(valid), dtype=\"int32\")\n",
    "    else:\n",
    "        valid['org_sorting'] = np.arange(len(valid), dtype=\"int32\")\n",
    "    df_tmp_m = valid[col + ['org_sorting']].merge(df_tmp, how='left', left_on=col, right_on=col).sort_values('org_sorting')\n",
    "    valid['TE_' + col_name] = df_tmp_m['TE_tmp'].fillna(mn).values\n",
    "    \n",
    "    valid = valid.drop('org_sorting', axis=1)\n",
    "    train = train.drop('kfold', axis=1)\n",
    "    train = train.drop('org_sorting', axis=1)\n",
    "    return(train, valid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_pd = df_train.to_pandas()\n",
    "df_valid_pd = df_valid.to_pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 44.3 s, sys: 23.8 s, total: 1min 8s\n",
      "Wall time: 1min 8s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df_train_pd, df_valid_pd = target_encode(df_train_pd, df_valid_pd, ['ts_weekday', 'ts_hour', 'cat_2', 'brand'], 'target', gpu=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 3.77 s, sys: 4.04 s, total: 7.81 s\n",
      "Wall time: 7.56 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "df_train, df_valid = target_encode(df_train, df_valid, ['ts_weekday', 'ts_hour', 'cat_2', 'brand'], 'target')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In our experiments, we achieve a speed up of 11.6x."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our implementation can be still improved. We will show a further optimized solution based on dask and dask_cudf."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We shutdown the kernel."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'status': 'ok', 'restart': False}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "app = IPython.Application.instance()\n",
    "app.kernel.do_shutdown(False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
